{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ce88749d-2bdc-4f15-9c67-70b8010d6720",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_docling import DoclingLoader\n",
    "from docling.chunking import HybridChunker\n",
    "from langchain_community.embeddings import LlamaCppEmbeddings\n",
    "from docling.document_converter import DocumentConverter"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "baafc7b0-ebd7-47c4-b7e2-7f713cc38c6d",
   "metadata": {},
   "source": [
    "# 加载兰德马克帮助文档"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "dd66c6ff-66f8-4e6f-8ce5-3ebf9c0cca87",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Error loading file /home/spike/Documents/help/webhelp/oxygen-webhelp/search/search-excludes.txt: Invalid input object: NoneType\n",
      "Error loading file /home/spike/Documents/help/oxygen-webhelp/search/search-excludes.txt: Invalid input object: NoneType\n"
     ]
    }
   ],
   "source": [
    "from langchain_community.document_loaders import DirectoryLoader\n",
    "from langchain_community.document_loaders import UnstructuredHTMLLoader\n",
    "from typing import List\n",
    "from langchain.schema import Document\n",
    "from typing import List, Union\n",
    "import chardet\n",
    "import os\n",
    "from wcmatch import glob\n",
    "\n",
    "# 加载所有 .html、.htm、.xhtml 文件（递归）\n",
    "loader = DirectoryLoader(\n",
    "    path=\"/home/spike/Documents/help\",\n",
    "    glob=['**/*.htm','**/*.html','**/*.md','**/*.txt'],  # 混合类型\n",
    "    loader_cls=UnstructuredHTMLLoader,\n",
    "    show_progress=False,\n",
    "    use_multithreading=True,\n",
    "    silent_errors = True\n",
    ")\n",
    "\n",
    "docs = loader.load()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9cc2a2b-33be-4ea2-991e-8a1175781aa4",
   "metadata": {},
   "source": [
    "# 加载DeepSeek Embeadding模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "16f9a2d7-e2fa-4277-8c8f-6e79018077cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no\n",
      "ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no\n",
      "ggml_cuda_init: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3060, compute capability 8.6, VMM: yes\n",
      "llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3060) - 11671 MiB free\n",
      "llama_model_loader: loaded meta data with 23 key-value pairs and 101 tensors from ./all-MiniLM-L6-v2.F32.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = bert\n",
      "llama_model_loader: - kv   1:                               general.name str              = all-MiniLM-L6-v2\n",
      "llama_model_loader: - kv   2:                           bert.block_count u32              = 6\n",
      "llama_model_loader: - kv   3:                        bert.context_length u32              = 512\n",
      "llama_model_loader: - kv   4:                      bert.embedding_length u32              = 384\n",
      "llama_model_loader: - kv   5:                   bert.feed_forward_length u32              = 1536\n",
      "llama_model_loader: - kv   6:                  bert.attention.head_count u32              = 12\n",
      "llama_model_loader: - kv   7:          bert.attention.layer_norm_epsilon f32              = 0.000000\n",
      "llama_model_loader: - kv   8:                          general.file_type u32              = 0\n",
      "llama_model_loader: - kv   9:                      bert.attention.causal bool             = false\n",
      "llama_model_loader: - kv  10:                          bert.pooling_type u32              = 1\n",
      "llama_model_loader: - kv  11:            tokenizer.ggml.token_type_count u32              = 2\n",
      "llama_model_loader: - kv  12:                tokenizer.ggml.bos_token_id u32              = 101\n",
      "llama_model_loader: - kv  13:                tokenizer.ggml.eos_token_id u32              = 102\n",
      "llama_model_loader: - kv  14:                       tokenizer.ggml.model str              = bert\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.tokens arr[str,30522]   = [\"[PAD]\", \"[unused0]\", \"[unused1]\", \"...\n",
      "llama_model_loader: - kv  16:                      tokenizer.ggml.scores arr[f32,30522]   = [-1000.000000, -1000.000000, -1000.00...\n",
      "llama_model_loader: - kv  17:                  tokenizer.ggml.token_type arr[i32,30522]   = [3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 100\n",
      "llama_model_loader: - kv  19:          tokenizer.ggml.seperator_token_id u32              = 102\n",
      "llama_model_loader: - kv  20:            tokenizer.ggml.padding_token_id u32              = 0\n",
      "llama_model_loader: - kv  21:                tokenizer.ggml.cls_token_id u32              = 101\n",
      "llama_model_loader: - kv  22:               tokenizer.ggml.mask_token_id u32              = 103\n",
      "llama_model_loader: - type  f32:  101 tensors\n",
      "print_info: file format = GGUF V3 (latest)\n",
      "print_info: file type   = all F32\n",
      "print_info: file size   = 86.08 MiB (32.00 BPW) \n",
      "init_tokenizer: initializing tokenizer for type 3\n",
      "load: control token:    101 '[CLS]' is not marked as EOG\n",
      "load: control token:    103 '[MASK]' is not marked as EOG\n",
      "load: control token:      0 '[PAD]' is not marked as EOG\n",
      "load: control token:    100 '[UNK]' is not marked as EOG\n",
      "load: control token:    102 '[SEP]' is not marked as EOG\n",
      "load: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect\n",
      "load: special tokens cache size = 5\n",
      "load: token to piece cache size = 0.2032 MB\n",
      "print_info: arch             = bert\n",
      "print_info: vocab_only       = 0\n",
      "print_info: n_ctx_train      = 512\n",
      "print_info: n_embd           = 384\n",
      "print_info: n_layer          = 6\n",
      "print_info: n_head           = 12\n",
      "print_info: n_head_kv        = 12\n",
      "print_info: n_rot            = 32\n",
      "print_info: n_swa            = 0\n",
      "print_info: n_embd_head_k    = 32\n",
      "print_info: n_embd_head_v    = 32\n",
      "print_info: n_gqa            = 1\n",
      "print_info: n_embd_k_gqa     = 384\n",
      "print_info: n_embd_v_gqa     = 384\n",
      "print_info: f_norm_eps       = 1.0e-12\n",
      "print_info: f_norm_rms_eps   = 0.0e+00\n",
      "print_info: f_clamp_kqv      = 0.0e+00\n",
      "print_info: f_max_alibi_bias = 0.0e+00\n",
      "print_info: f_logit_scale    = 0.0e+00\n",
      "print_info: f_attn_scale     = 0.0e+00\n",
      "print_info: n_ff             = 1536\n",
      "print_info: n_expert         = 0\n",
      "print_info: n_expert_used    = 0\n",
      "print_info: causal attn      = 0\n",
      "print_info: pooling type     = 1\n",
      "print_info: rope type        = 2\n",
      "print_info: rope scaling     = linear\n",
      "print_info: freq_base_train  = 10000.0\n",
      "print_info: freq_scale_train = 1\n",
      "print_info: n_ctx_orig_yarn  = 512\n",
      "print_info: rope_finetuned   = unknown\n",
      "print_info: ssm_d_conv       = 0\n",
      "print_info: ssm_d_inner      = 0\n",
      "print_info: ssm_d_state      = 0\n",
      "print_info: ssm_dt_rank      = 0\n",
      "print_info: ssm_dt_b_c_rms   = 0\n",
      "print_info: model type       = 22M\n",
      "print_info: model params     = 22.57 M\n",
      "print_info: general.name     = all-MiniLM-L6-v2\n",
      "print_info: vocab type       = WPM\n",
      "print_info: n_vocab          = 30522\n",
      "print_info: n_merges         = 0\n",
      "print_info: BOS token        = 101 '[CLS]'\n",
      "print_info: EOS token        = 102 '[SEP]'\n",
      "print_info: UNK token        = 100 '[UNK]'\n",
      "print_info: SEP token        = 102 '[SEP]'\n",
      "print_info: PAD token        = 0 '[PAD]'\n",
      "print_info: MASK token       = 103 '[MASK]'\n",
      "print_info: LF token         = 0 '[PAD]'\n",
      "print_info: EOG token        = 102 '[SEP]'\n",
      "print_info: max token length = 21\n",
      "load_tensors: loading model tensors, this can take a while... (mmap = true)\n",
      "load_tensors: layer   0 assigned to device CPU\n",
      "load_tensors: layer   1 assigned to device CPU\n",
      "load_tensors: layer   2 assigned to device CPU\n",
      "load_tensors: layer   3 assigned to device CPU\n",
      "load_tensors: layer   4 assigned to device CPU\n",
      "load_tensors: layer   5 assigned to device CPU\n",
      "load_tensors: layer   6 assigned to device CPU\n",
      "load_tensors: tensor 'token_embd.weight' (f32) (and 100 others) cannot be used with preferred buffer type CPU_AARCH64, using CPU instead\n",
      "load_tensors: offloading 0 repeating layers to GPU\n",
      "load_tensors: offloaded 0/7 layers to GPU\n",
      "load_tensors:   CPU_Mapped model buffer size =    86.08 MiB\n",
      "...............................\n",
      "llama_init_from_model: n_seq_max     = 1\n",
      "llama_init_from_model: n_ctx         = 512\n",
      "llama_init_from_model: n_ctx_per_seq = 512\n",
      "llama_init_from_model: n_batch       = 512\n",
      "llama_init_from_model: n_ubatch      = 512\n",
      "llama_init_from_model: flash_attn    = 0\n",
      "llama_init_from_model: freq_base     = 10000.0\n",
      "llama_init_from_model: freq_scale    = 1\n",
      "llama_kv_cache_init: kv_size = 512, offload = 1, type_k = 'f16', type_v = 'f16', n_layer = 6, can_shift = 1\n",
      "llama_kv_cache_init: layer 0: n_embd_k_gqa = 384, n_embd_v_gqa = 384\n",
      "llama_kv_cache_init: layer 1: n_embd_k_gqa = 384, n_embd_v_gqa = 384\n",
      "llama_kv_cache_init: layer 2: n_embd_k_gqa = 384, n_embd_v_gqa = 384\n",
      "llama_kv_cache_init: layer 3: n_embd_k_gqa = 384, n_embd_v_gqa = 384\n",
      "llama_kv_cache_init: layer 4: n_embd_k_gqa = 384, n_embd_v_gqa = 384\n",
      "llama_kv_cache_init: layer 5: n_embd_k_gqa = 384, n_embd_v_gqa = 384\n",
      "llama_kv_cache_init:        CPU KV buffer size =     4.50 MiB\n",
      "llama_init_from_model: KV self size  =    4.50 MiB, K (f16):    2.25 MiB, V (f16):    2.25 MiB\n",
      "llama_init_from_model:        CPU  output buffer size =     0.00 MiB\n",
      "llama_init_from_model:      CUDA0 compute buffer size =    17.31 MiB\n",
      "llama_init_from_model:  CUDA_Host compute buffer size =     3.50 MiB\n",
      "llama_init_from_model: graph nodes  = 221\n",
      "llama_init_from_model: graph splits = 100 (with bs=512), 1 (with bs=1)\n",
      "CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | \n",
      "Model metadata: {'tokenizer.ggml.mask_token_id': '103', 'tokenizer.ggml.padding_token_id': '0', 'tokenizer.ggml.seperator_token_id': '102', 'tokenizer.ggml.unknown_token_id': '100', 'tokenizer.ggml.model': 'bert', 'tokenizer.ggml.eos_token_id': '102', 'general.architecture': 'bert', 'bert.block_count': '6', 'bert.attention.layer_norm_epsilon': '0.000000', 'bert.context_length': '512', 'bert.feed_forward_length': '1536', 'bert.embedding_length': '384', 'tokenizer.ggml.cls_token_id': '101', 'tokenizer.ggml.token_type_count': '2', 'bert.attention.head_count': '12', 'tokenizer.ggml.bos_token_id': '101', 'general.file_type': '0', 'general.name': 'all-MiniLM-L6-v2', 'bert.attention.causal': 'false', 'bert.pooling_type': '1'}\n",
      "Using fallback chat format: llama-2\n"
     ]
    }
   ],
   "source": [
    "llama = LlamaCppEmbeddings(model_path=\"./all-MiniLM-L6-v2.F32.gguf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9710d094-c832-41be-88d8-c6c452d30891",
   "metadata": {},
   "source": [
    "# 使用Sqlite作为向量数据库"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "336bab4d-5673-4cff-98d8-1d3ed3c1f825",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "llama_perf_context_print:        load time =      32.47 ms\n",
      "llama_perf_context_print: prompt eval time =      30.99 ms /     7 tokens (    4.43 ms per token,   225.88 tokens per second)\n",
      "llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)\n",
      "llama_perf_context_print:       total time =      32.52 ms /     8 tokens\n"
     ]
    }
   ],
   "source": [
    "from langchain_community.vectorstores import SQLiteVec\n",
    "# Initialize with an embedding model\n",
    "vector_store = SQLiteVec(\n",
    "    connection=None, table=\"state_union\", db_file=\"docsstore.db\", embedding=llama\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "229979c9-dcf9-4e75-9f3d-094d5a7fb018",
   "metadata": {},
   "source": [
    "# 每500条文档写入一次向量数据库"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e2bd57f3-fe1b-40e0-8b38-479c5e600f2e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "llama_perf_context_print:        load time =      32.47 ms\n",
      "llama_perf_context_print: prompt eval time =    3364.00 ms / 91886 tokens (    0.04 ms per token, 27314.52 tokens per second)\n",
      "llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)\n",
      "llama_perf_context_print:       total time =    3563.47 ms / 91887 tokens\n",
      "llama_perf_context_print:        load time =      32.47 ms\n",
      "llama_perf_context_print: prompt eval time =    5760.34 ms / 167522 tokens (    0.03 ms per token, 29081.95 tokens per second)\n",
      "llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)\n",
      "llama_perf_context_print:       total time =    6125.15 ms / 167523 tokens\n",
      "llama_perf_context_print:        load time =      32.47 ms\n",
      "llama_perf_context_print: prompt eval time =    3547.97 ms / 106168 tokens (    0.03 ms per token, 29923.62 tokens per second)\n",
      "llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)\n",
      "llama_perf_context_print:       total time =    3745.38 ms / 106169 tokens\n",
      "llama_perf_context_print:        load time =      32.47 ms\n",
      "llama_perf_context_print: prompt eval time =    4880.48 ms / 155867 tokens (    0.03 ms per token, 31936.79 tokens per second)\n",
      "llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)\n",
      "llama_perf_context_print:       total time =    5455.11 ms / 155868 tokens\n",
      "llama_perf_context_print:        load time =      32.47 ms\n",
      "llama_perf_context_print: prompt eval time =    4269.45 ms / 138184 tokens (    0.03 ms per token, 32365.73 tokens per second)\n",
      "llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)\n",
      "llama_perf_context_print:       total time =    4513.17 ms / 138185 tokens\n",
      "llama_perf_context_print:        load time =      32.47 ms\n",
      "llama_perf_context_print: prompt eval time =    4843.79 ms / 158023 tokens (    0.03 ms per token, 32623.83 tokens per second)\n",
      "llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)\n",
      "llama_perf_context_print:       total time =    5125.95 ms / 158024 tokens\n",
      "llama_perf_context_print:        load time =      32.47 ms\n",
      "llama_perf_context_print: prompt eval time =    3657.89 ms / 120001 tokens (    0.03 ms per token, 32806.04 tokens per second)\n",
      "llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)\n",
      "llama_perf_context_print:       total time =    3853.99 ms / 120002 tokens\n",
      "llama_perf_context_print:        load time =      32.47 ms\n",
      "llama_perf_context_print: prompt eval time =    4288.21 ms / 139912 tokens (    0.03 ms per token, 32627.13 tokens per second)\n",
      "llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)\n",
      "llama_perf_context_print:       total time =    4535.54 ms / 139913 tokens\n",
      "llama_perf_context_print:        load time =      32.47 ms\n",
      "llama_perf_context_print: prompt eval time =    4043.72 ms / 129084 tokens (    0.03 ms per token, 31922.09 tokens per second)\n",
      "llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)\n",
      "llama_perf_context_print:       total time =    4267.37 ms / 129085 tokens\n",
      "llama_perf_context_print:        load time =      32.47 ms\n",
      "llama_perf_context_print: prompt eval time =    4066.98 ms / 128505 tokens (    0.03 ms per token, 31597.17 tokens per second)\n",
      "llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)\n",
      "llama_perf_context_print:       total time =    4275.26 ms / 128506 tokens\n",
      "llama_perf_context_print:        load time =      32.47 ms\n",
      "llama_perf_context_print: prompt eval time =    3677.24 ms / 109873 tokens (    0.03 ms per token, 29879.22 tokens per second)\n",
      "llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)\n",
      "llama_perf_context_print:       total time =    3884.89 ms / 109874 tokens\n",
      "llama_perf_context_print:        load time =      32.47 ms\n",
      "llama_perf_context_print: prompt eval time =    4686.51 ms / 131338 tokens (    0.04 ms per token, 28024.70 tokens per second)\n",
      "llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)\n",
      "llama_perf_context_print:       total time =    4984.62 ms / 131339 tokens\n",
      "llama_perf_context_print:        load time =      32.47 ms\n",
      "llama_perf_context_print: prompt eval time =    3865.86 ms / 115800 tokens (    0.03 ms per token, 29954.55 tokens per second)\n",
      "llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)\n",
      "llama_perf_context_print:       total time =    4082.67 ms / 115801 tokens\n",
      "llama_perf_context_print:        load time =      32.47 ms\n",
      "llama_perf_context_print: prompt eval time =    1764.55 ms / 56163 tokens (    0.03 ms per token, 31828.58 tokens per second)\n",
      "llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)\n",
      "llama_perf_context_print:       total time =    1861.95 ms / 56164 tokens\n"
     ]
    }
   ],
   "source": [
    "from uuid import uuid4\n",
    "#uuids = [str(uuid4()) for _ in range(len(docs))]\n",
    "chunk = 500\n",
    "for i in range(0, len(docs), chunk):\n",
    "    #vector_store.add_documents(documents=docs[i:i+chunk], ids=uuids[i:i+chunk])\n",
    "    vector_store.add_documents(documents=docs[i:i+chunk])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6b26ef0-b2fd-4826-9109-576b5aa15916",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_text_splitters import CharacterTextSplitter\n",
    "text_splitter = CharacterTextSplitter.from_tiktoken_encoder(\n",
    "    encoding_name=\"cl100k_base\", chunk_size=20, chunk_overlap=0\n",
    ")\n",
    "texts = text_splitter.split_text(docs[0].page_content)\n",
    "print(texts[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aea899cb-2554-4be8-b53d-16fda551a77e",
   "metadata": {},
   "source": [
    "%pip install sqlite-vec\n",
    "%pip install wcmatch"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
